#ifndef ROBOTVERIFIER_H_
#define ROBOTVERIFIER_H_
#include <iostream>
#include <fstream>


class RobotVerifier
{
public:
	RobotVerifier();
	virtual ~RobotVerifier();
	
	/* TODO: Fetch robots.txt */
	
	static bool analyseRobotsTXT(std::string baseUrl, std::ifstream* file);
	static bool analyseRobotsTXT(std::string baseUrl,std::string domain);
	static int fetchRobotsTXT(std::string baseDomain);
};

#endif /*ROBOTVERIFIER_H_*/
